package com.laivi.basic.module.crawler;

import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import com.laivi.basic.model.crawler.LinkParser;
import com.laivi.basic.model.crawler.LinkSet;

/**
 * @author laivi.zhu@gmail.com
 * @date 2014-5-20
 * description:
 */
public class CrawlerThread implements Runnable {
	
	private ExecutorService pool;
	private LinkSet links;
	private Parsers parser;
	private CountDownLatch latch; 
	
	public CrawlerThread(Parsers parser,CountDownLatch latch){
		this.parser=parser;
		this.latch=latch;
		links=new LinkSet();
		pool=Executors.newFixedThreadPool(10);
		for (int i = 0; i < parser.getSeeds().length; i++) {
			links.addUnvisitedUrl(parser.getSeeds()[i]);
        }
	}

	@Override
	public void run() {
		while(!links.unVisitedUrlsEmpty()){
			if(parser.getMax()==0||parser.getParserNum()<parser.getMax()){
				try{
					String visitUrl = links.unVisitedUrlDeQueue();
	                if (visitUrl == null) {
	                    continue;
	                }
	                links.addVisitedUrl(visitUrl);
	                for(String seed:LinkParser.extractLinks(visitUrl, parser.getParserLinkFilter())){
	                	links.addUnvisitedUrl(seed);
	                }
	                
	                for(String link:LinkParser.extractLinks(visitUrl,parser.getLinkFilter())){
	                	pool.execute(new ParserThread(parser,links,link,parser.getKey()));
	                }
				}catch(Exception e){
					e.printStackTrace();
				}
			}
    	}
		pool.shutdown();
		latch.countDown();
	}
}
